namespace SimpleWebServices.Net
{
    using System;
    using System.Collections.Generic;
    using System.Collections.ObjectModel;
    using System.Globalization;
    using System.Net;
    using System.Net.Mail;
    using System.Net.Mime;
    using System.Text;
    using System.Xml;
    using System.Xml.XPath;

    public sealed class Spider
    {
        private const string atom = "http://www.w3.org/2005/Atom";
        private const string dc = "http://purl.org/dc/elements/1.1/";
        private const string rdf = "http://www.w3.org/1999/02/22-rdf-syntax-ns#";
        private const string rss = "http://purl.org/rss/1.0/modules/";
        private const string sitemaps = "http://www.sitemaps.org/schemas/sitemap/0.9";
        private const string xhtml = "http://www.w3.org/1999/xhtml";
        private const string xlink = "http://www.w3.org/1999/xlink";

        private string _domain;

        public Spider(string domain)
        {
            this.Domain = domain;
            this.Visited = new Collection<AbsoluteUri>();
        }

        private Collection<AbsoluteUri> Visited
        {
            get;
            set;
        }

        private string Domain
        {
            get
            {
                return this._domain;
            }

            set
            {
                if (null == value)
                {
                    throw new ArgumentNullException("value");
                }
                else if (0 == value.Length)
                {
                    throw new ArgumentOutOfRangeException("value");
                }

                this._domain = value;
            }
        }

        public bool Crawl(AbsoluteUri uri)
        {
            return this.Crawl(uri, null as ContentType);
        }

        private static void CheckXhtmlStructure(Response response, XPathNavigator navigator)
        {
            Spider.CheckHtmlStructure(response, navigator, "xhtml:");
        }

        private static void CheckHtmlStructure(Response response, XPathNavigator navigator)
        {
            Spider.CheckHtmlStructure(response, navigator, string.Empty);
        }

        private static string CheckHtmlAttributeFor(XPathNavigator navigator, XmlNamespaceManager namespaces, string prefix)
        {
            StringBuilder buffer = new StringBuilder();

            XPathNodeIterator iterator = navigator.Select(string.Format(CultureInfo.InvariantCulture, "//{0}label/@for", prefix), namespaces);
            while (iterator.MoveNext())
            {
                if ((bool)navigator.Evaluate(string.Format(CultureInfo.InvariantCulture, "0 = count(//{0}button[@id='{1}'] | //{0}input[@id='{1}'] | //{0}select[@id='{1}'] | //{0}textarea[@id='{1}'])", prefix, iterator.Current.Value), namespaces))
                {
                    buffer.AppendLine("<label for='" + iterator.Current.Value + "'> does not have a matching control element (<button>, <input>, <select> or <textarea>).");
                }
            }

            return buffer.ToString();
        }

        private static string CheckHtmlAttributeNotEmpty(XPathNavigator navigator, XmlNamespaceManager namespaces)
        {
            StringBuilder buffer = new StringBuilder();

            foreach (string name in new string[]
                {
                    "abbr",
                    "accept-charset",
                    "accept",
                    "accesskey",
                    "action",
                    "alt",
                    "archive",
                    "axis",
                    "border",
                    "cellpadding",
                    "cellspacing",
                    "char",
                    "charoff",
                    "charset",
                    "cite",
                    "classid",
                    "codebase",
                    "codetype",
                    "cols",
                    "colspan",
                    "content",
                    "coords",
                    "data",
                    "datetime",
                    "for",
                    "frame",
                    "headers",
                    "height",
                    "href",
                    "hreflang",
                    "http-equiv",
                    "id",
                    "label",
                    "longdesc",
                    "marginheight",
                    "marginwidth",
                    "maxlength",
                    "media",
                    "name",
                    "onblur",
                    "onchange",
                    "onclick",
                    "ondblclick",
                    "onfocus",
                    "onkeydown",
                    "onkeypress",
                    "onkeyup",
                    "onload",
                    "onmousedown",
                    "onmousemove",
                    "onmouseout",
                    "onmouseover",
                    "onmouseup",
                    "onreset",
                    "onselect",
                    "onsubmit",
                    "onunload",
                    "profile",
                    "rel",
                    "rev",
                    "rows",
                    "rowspan",
                    "rules",
                    "scheme",
                    "scope",
                    "shape",
                    "size",
                    "span",
                    "src",
                    "style",
                    "summary",
                    "tabindex",
                    "target",
                    "title",
                    "type",
                    "usemap",
                    "width"
                })
            {
                buffer.Append(Spider.CheckHtmlAttributeNotEmpty(navigator, namespaces, name));
            }

            return buffer.ToString();
        }

        private static string CheckHtmlAttributeNotEmpty(XPathNavigator navigator, XmlNamespaceManager namespaces, string attribute)
        {
            string result = null;

            if ((bool)navigator.Evaluate(string.Concat("0 != count(//*[@", attribute, "=''])"), namespaces))
            {
                result = string.Concat("An empty ", attribute, " attribute was found.", Environment.NewLine);
            }

            return result;
        }

        private static string CheckHtmlAttributeRequired(XPathNavigator navigator, XmlNamespaceManager namespaces, string prefix)
        {
            StringBuilder buffer = new StringBuilder();

            foreach (string xpath in new string[]
                {
                    "//{0}area[not(@alt)]",
                    "//{0}bdo[not(@dir)]",
                    "//{0}form[not(@action)]",
                    "//{0}img[not(@src)][not(@alt)]",
                    "//{0}input[@type='checkbox'][not(@value)]",
                    "//{0}input[@type='radio'][not(@value)]",
                    "//{0}input[@type='text'][not(@maxlength)]",
                    "//{0}link[not(@href)]", /// this is opinion, not specification
                    "//{0}map[not(@name)]",
                    "//{0}meta[not(@content)]",
                    "//{0}optgroup[not(@label)]",
                    "//{0}param[not(@name)]",
                    "//{0}script[not(@type)]",
                    "//{0}style[not(@type)]",
                    "//{0}textarea[not(@cols)][not(@rows)]"
                })
            {
                buffer.Append(Spider.CheckHtmlAttributeRequired(navigator, namespaces, prefix, xpath));
            }

            return buffer.ToString();
        }

        private static string CheckHtmlAttributeRequired(XPathNavigator navigator, XmlNamespaceManager namespaces, string prefix, string xpath)
        {
            string result = null;

            xpath = string.Format(CultureInfo.InvariantCulture, string.Concat("0 != count(", xpath, ")"), prefix);
            if ((bool)navigator.Evaluate(xpath, namespaces))
            {
                result = string.Concat("Required attributes were not found: ", xpath, Environment.NewLine);
            }

            return result;
        }

        private static string CheckHtmlAttributeTypeCharacter(XPathNavigator navigator, XmlNamespaceManager namespaces)
        {
            StringBuilder buffer = new StringBuilder();

            if ((bool)navigator.Evaluate("0 != count(//*[@accesskey][1!=string-length(@accesskey)])", namespaces))
            {
                buffer.AppendLine("An accesskey attribute was found which was not exactly one character long.");
            }

            if ((bool)navigator.Evaluate("0 != count(//*[@char][1!=string-length(@char)])", namespaces))
            {
                buffer.AppendLine("An char attribute was found which was not exactly one character long.");
            }

            return buffer.ToString();
        }

        private static string CheckHtmlAttributeValue(XPathNavigator navigator, XmlNamespaceManager namespaces)
        {
            StringBuilder buffer = new StringBuilder();

            foreach (string name in new string[]
                {
                    "checked",
                    "declare",
                    "defer",
                    "disabled",
                    "ismap",
                    "multiple",
                    "nohref",
                    "noresize",
                    "readonly",
                    "selected"
                })
            {
                buffer.Append(Spider.CheckHtmlAttributeValue(navigator, namespaces, name));
            }

            buffer.Append(Spider.CheckHtmlAttributeValue(navigator, namespaces, "align", new string[] { "left", "center", "right", "justify", "char" }));
            buffer.Append(Spider.CheckHtmlAttributeValue(navigator, namespaces, "dir", new string[] { "ltr", "rtl" }));
            buffer.Append(Spider.CheckHtmlAttributeValue(navigator, namespaces, "enctype", new string[] { "application/x-www-form-urlencoded" }));
            buffer.Append(Spider.CheckHtmlAttributeValue(navigator, namespaces, "frameborder", new string[] { "0", "1" }));
            buffer.Append(Spider.CheckHtmlAttributeValue(navigator, namespaces, "method", new string[] { "get", "post" }));
            buffer.Append(Spider.CheckHtmlAttributeValue(navigator, namespaces, "scrolling", new string[] { "yes", "no", "auto" }));
            buffer.Append(Spider.CheckHtmlAttributeValue(navigator, namespaces, "valign", new string[] { "top", "middle", "bottom", "baseline" }));
            buffer.Append(Spider.CheckHtmlAttributeValue(navigator, namespaces, "valuetype", new string[] { "data", "ref", "object" }));

            return buffer.ToString();
        }

        private static string CheckHtmlAttributeValue(XPathNavigator navigator, XmlNamespaceManager namespaces, string attribute)
        {
            return Spider.CheckHtmlAttributeValue(navigator, namespaces, attribute, new string[] { attribute });
        }

        private static string CheckHtmlAttributeValue(XPathNavigator navigator, XmlNamespaceManager namespaces, string attribute, string[] values)
        {
            XPathNodeIterator iterator = navigator.Select("//@" + attribute, namespaces);
            while (iterator.MoveNext())
            {
                bool equals = false;
                foreach (string value in values)
                {
                    if (string.Equals(value, iterator.Current.Value, StringComparison.OrdinalIgnoreCase))
                    {
                        equals = true;
                        break;
                    }
                }

                if (!equals)
                {
                    return string.Concat("At least one ", attribute, " attribute was found with an invalid value.", Environment.NewLine);
                }
            }

            return null;
        }

        private static string CheckHtmlDeprecated(XPathNavigator navigator, XmlNamespaceManager namespaces, string prefix)
        {
            StringBuilder buffer = new StringBuilder();

            foreach (string xpath in new string[]
                {
                    "//{0}applet",
                    "//{0}basefont",
                    "//{0}center",
                    "//{0}dir",
                    "//{0}font",
                    "//{0}isindex",
                    "//{0}menu",
                    "//{0}s",
                    "//{0}strike",
                    "//{0}u",
                    "//{0}caption/@align",
                    "//{0}iframe/@align",
                    "//{0}img/@align",
                    "//{0}input/@align",
                    "//{0}object/@align",
                    "//{0}legend/@align",
                    "//{0}table/@align",
                    "//{0}hr/@align",
                    "//{0}div/@align",
                    "//{0}h1/@align",
                    "//{0}h2/@align",
                    "//{0}h3/@align",
                    "//{0}h4/@align",
                    "//{0}h5/@align",
                    "//{0}h6/@align",
                    "//{0}html/@version",
                    "//{0}p/@align",
                    "//{0}img/@border",
                    "//{0}hr/@size",
                    "//{0}hr/@width",
                    "//{0}li/@type",
                    "//{0}li/@value",
                    "//{0}object/@border",
                    "//{0}ol/@type",
                    "//{0}pre/@width",
                    "//{0}td/@height",
                    "//{0}td/@width",
                    "//{0}th/@height",
                    "//{0}th/@width",
                    "//{0}ul/@type",
                    "//@alink",
                    "//@background",
                    "//@bgcolor",
                    "//@clear",
                    "//@color",
                    "//@compact",
                    "//@face",
                    "//@hspace",
                    "//@language",
                    "//@link",
                    "//@noshade",
                    "//@nowrap",
                    "//@prompt",
                    "//@start",
                    "//@text",
                    "//@vlink",
                    "//@vspace"
                })
            {
                buffer.Append(Spider.CheckHtmlDeprecated(navigator, namespaces, prefix, xpath));
            }

            return buffer.ToString();
        }

        private static string CheckHtmlDeprecated(XPathNavigator navigator, XmlNamespaceManager namespaces, string prefix, string xpath)
        {
            string result = null;

            xpath = string.Format(CultureInfo.InvariantCulture, string.Concat("0 != count(", xpath, ")"), prefix);
            if ((bool)navigator.Evaluate(xpath, namespaces))
            {
                result = string.Concat("Deprecated html was found: ", xpath, Environment.NewLine);
            }

            return result;
        }

        private static string CheckHtmlHeading1(XPathNavigator navigator, XmlNamespaceManager namespaces, string prefix)
        {
            StringBuilder buffer = new StringBuilder();

            if ((bool)navigator.Evaluate(string.Format(CultureInfo.InvariantCulture, "0 = count(//{0}h1)", prefix), namespaces))
            {
                buffer.AppendLine("An <h1> is required.");
            }
            else if ((bool)navigator.Evaluate(string.Format(CultureInfo.InvariantCulture, "1 != count(//{0}h1)", prefix), namespaces))
            {
                buffer.AppendLine("Only one <h1> is permitted.");
            }

            return buffer.ToString();
        }

        private static string CheckHtmlTitle(XPathNavigator navigator, XmlNamespaceManager namespaces, string prefix)
        {
            StringBuilder buffer = new StringBuilder();

            if ((bool)navigator.Evaluate(string.Format(CultureInfo.InvariantCulture, "0 = count(//{0}title)", prefix), namespaces))
            {
                buffer.AppendLine("A <title> is required.");
            }
            else if ((bool)navigator.Evaluate(string.Format(CultureInfo.InvariantCulture, "1 = count(//{0}title)", prefix), namespaces))
            {
                if ((bool)navigator.Evaluate(string.Format(CultureInfo.InvariantCulture, "0 = count(/{0}html/{0}head/{0}title)", prefix), namespaces))
                {
                    buffer.AppendLine("<title> must be a child of <head>.");
                }
            }
            else
            {
                buffer.AppendLine("Only one <title> is permitted.");
            }

            return buffer.ToString();
        }

        private static void CheckHtmlStructure(Response response, XPathNavigator navigator, string prefix)
        {
            StringBuilder buffer = new StringBuilder();

            XmlNamespaceManager namespaces = new XmlNamespaceManager(navigator.NameTable);
            namespaces.AddNamespace("xhtml", xhtml);

            buffer.Append(Spider.CheckHtmlTitle(navigator, namespaces, prefix));
            buffer.Append(Spider.CheckHtmlHeading1(navigator, namespaces, prefix));

            /// Attributes, see http://www.w3.org/TR/html401/index/attributes.html

            if ((bool)navigator.Evaluate("0 != count(//*[@lang=''] | //*[@xml:lang=''])", namespaces))
            {
                buffer.AppendLine("An empty lang or xml:lang attribute was found.");
            }

            buffer.Append(Spider.CheckHtmlAttributeTypeCharacter(navigator, namespaces));
            buffer.Append(Spider.CheckHtmlAttributeNotEmpty(navigator, namespaces));
            buffer.Append(Spider.CheckHtmlAttributeValue(navigator, namespaces));
            buffer.Append(Spider.CheckHtmlAttributeRequired(navigator, namespaces, prefix));
            buffer.Append(Spider.CheckHtmlDeprecated(navigator, namespaces, prefix));
            buffer.Append(Spider.CheckHtmlAttributeFor(navigator, namespaces, prefix));

            if (0 != buffer.Length)
            {
                throw new HttpTestException(response, "HTML structural issues were encountered:" + Environment.NewLine + buffer.ToString());
            }
        }

        private static void CheckXhtmlAccessibility(Response response, XPathNavigator navigator)
        {
            Spider.CheckHtmlAccessibility(response, navigator, "xhtml:");
        }

        private static void CheckHtmlAccessibility(Response response, XPathNavigator navigator)
        {
            Spider.CheckHtmlAccessibility(response, navigator, string.Empty);
        }

        private static void CheckHtmlAccessibility(Response response, XPathNavigator navigator, string prefix)
        {
            StringBuilder buffer = new StringBuilder();

            /// http://www.w3.org/TR/WCAG10/full-checklist.html

            bool xml = null != response as XmlResponse;
            XmlNamespaceManager namespaces = new XmlNamespaceManager(navigator.NameTable);
            namespaces.AddNamespace("xhtml", xhtml);

            /// [Priority 1]

            if ((bool)navigator.Evaluate(string.Format(CultureInfo.InvariantCulture, "0 != count(//{0}img[not(@alt)][not(@longdesc)])", prefix), namespaces))
            {
                buffer.AppendLine(
                    "<img> elements without either an 'alt' or 'longdesc' attribute was found. " +
                    "\"Provide a text equivalent for every non-text element (e.g., via 'alt', 'longdesc', or in element content). [Priority 1]\" " +
                    "http://www.w3.org/TR/WCAG10/wai-pageauth.html#tech-text-equivalent");
            }

            if ((bool)navigator.Evaluate(string.Format(CultureInfo.InvariantCulture, "0 = count(/{0}html[@lang])", prefix), namespaces))
            {
                buffer.AppendLine(
                    "<html> does not have a 'lang' attribute. " +
                    "\"Clearly identify changes in the natural language of a document's text. [Priority 1]\" " +
                    "http://www.w3.org/TR/WCAG10/wai-pageauth.html#tech-identify-changes");
            }

            if (xml && (bool)navigator.Evaluate(string.Format(CultureInfo.InvariantCulture, "0 = count(/{0}html[@xml:lang])", prefix), namespaces))
            {
                buffer.AppendLine(
                    "<html> does not have an 'xml:lang' attribute. " +
                    "\"Clearly identify changes in the natural language of a document's text. [Priority 1]\" " +
                    "http://www.w3.org/TR/WCAG10/wai-pageauth.html#tech-identify-changes");
            }

            /// [Priority 2]

            if ((bool)navigator.Evaluate(string.Format(CultureInfo.InvariantCulture, "0 != count(//{0}b | //{0}i)", prefix), namespaces))
            {
                buffer.AppendLine(
                    "The <b> and <i> elements should not be used; they are used to create a visual presentation effect. " +
                    "\"Use style sheets to control layout and presentation. [Priority 2]\" " +
                    "http://www.w3.org/TR/WCAG10-TECHS/#tech-style-sheets");
            }

            if ((bool)navigator.Evaluate(string.Format(CultureInfo.InvariantCulture, "0 != count(//{0}meta[@http-equiv='refresh'])", prefix), namespaces))
            {
                buffer.AppendLine(
                    "<meta http-equiv='refresh'> elements were found. " +
                    "\"Until user agents provide the ability to stop the refresh, do not create periodically auto-refreshing pages. [Priority 2]\" " +
                    "http://www.w3.org/TR/WCAG10-TECHS/#tech-no-periodic-refresh");
            }

            if ((bool)navigator.Evaluate(string.Format(CultureInfo.InvariantCulture, "0 != count(//{0}a[@target])", prefix), namespaces))
            {
                buffer.AppendLine(
                    "<a> elements with a 'target' attribute were found. " +
                    "\"Until user agents allow users to turn off spawned windows, " +
                    "do not cause pop-ups or other windows to appear and do not change the current window without informing the user. [Priority 2]\" " +
                    "http://www.w3.org/TR/WCAG10-HTML-TECHS/#anchors-targets");
            }

            /// [Priority 3]

            if ((bool)navigator.Evaluate(string.Format(CultureInfo.InvariantCulture, "0 != count(//{0}abbr[not(@title)] | //{0}acronym[not(@title)])", prefix), namespaces))
            {
                buffer.AppendLine(
                    "<abbr> or <acronym> elements either without a 'title' were found. " +
                    "\"Specify the expansion of each abbreviation or acronym in a document where it first occurs. [Priority 3]\" " +
                    "http://www.w3.org/TR/WCAG10-TECHS/#tech-expand-abbr");
            }

            if ((bool)navigator.Evaluate(string.Format(CultureInfo.InvariantCulture, "0 != count(//{0}table[not(@summary)])", prefix), namespaces))
            {
                buffer.AppendLine(
                    "<table> elements without a 'summary' were found. " +
                    "\"Provide summaries for tables. [Priority 3]\" " +
                    "http://www.w3.org/TR/WCAG10-HTML-TECHS/#table-summary-info");
            }

            /// Also

            if ((bool)navigator.Evaluate(string.Format(CultureInfo.InvariantCulture, "0 != count(//{0}table[not(@title)][not({0}caption)] | //{0}table[not(@title)]/{0}caption[not(text())])", prefix), namespaces))
            {
                buffer.AppendLine(
                    "<table> elements without a <caption> or 'title' attribute was found. " +
                    "\"Provide a caption via the CAPTION element. " +
                    "If a CAPTION is not provided, use the 'title' attribute on the TABLE element to describe the nature of the table in a few words.\" " +
                    "http://www.w3.org/TR/WCAG10-HTML-TECHS/#table-summary-info");
            }

            buffer.Append(Spider.CheckHtmlAccessibilityOpinion(navigator, namespaces, prefix));

            if (0 != buffer.Length)
            {
                throw new HttpTestException(response, "HTML accessibility issues were encountered:" + Environment.NewLine + buffer.ToString());
            }
        }

        private static string CheckHtmlAccessibilityOpinion(XPathNavigator navigator, XmlNamespaceManager namespaces, string prefix)
        {
            StringBuilder buffer = new StringBuilder();

            foreach (string xpath in new string[]
                {
                    "//{0}a[not(@rel)]",
                    "//{0}a[not(@title)]",
                    "//{0}button[not(@name)]",
                    "//{0}button[not(@title)]",
                    "//{0}button[not(@type)]",
                    "//{0}div[not(@id)][not(@class)]",
                    "//{0}fieldset[not({0}legend)]",
                    "//{0}legend[not(normalize-space(.))]",
                    "//{0}h1[not(normalize-space(.))]",
                    "//{0}h2[not(normalize-space(.))]",
                    "//{0}h3[not(normalize-space(.))]",
                    "//{0}h4[not(normalize-space(.))]",
                    "//{0}h5[not(normalize-space(.))]",
                    "//{0}h6[not(normalize-space(.))]",
                    "//{0}img[not(@height)] | //{0}img[not(@width)]",
                    "//{0}input[not(@type='submit')][not(@name)]",
                    "//{0}input[not(@type)]",
                    "//{0}input[not(@type='hidden')][not(@title)]",
                    "//{0}label[not(@title)]",
                    "//{0}select[not(@name)]",
                    "//{0}select[not(@title)]",
                    "//{0}textarea[not(@name)]",
                    "//{0}textarea[not(@title)]"
                })
            {
                buffer.Append(Spider.CheckHtmlAttributeRequired(navigator, namespaces, prefix, xpath));
            }

            return buffer.ToString();
        }

        private static AbsoluteUri ResolveBaseUri(AbsoluteUri uri, XPathNavigator navigator)
        {
            if (null != navigator)
            {
                XmlNamespaceManager namespaces = new XmlNamespaceManager(navigator.NameTable);
                namespaces.AddNamespace("xml", "http://www.w3.org/XML/1998/namespace");

                string value = (string)navigator.Evaluate("string(//@xml:base)", namespaces);
                if (!string.IsNullOrEmpty(value)
                    && (value.StartsWith("http://", StringComparison.OrdinalIgnoreCase) || value.StartsWith("https://", StringComparison.OrdinalIgnoreCase)))
                {
                    uri = value;
                }
            }

            return uri;
        }

        private bool Crawl(AbsoluteUri uri, ContentType type)
        {
            if (!this.Visited.Contains(uri))
            {
                HttpRequestDefinition definition = new HttpRequestDefinition(uri);
                if (null != type)
                {
                    definition.Accept = type.MediaType;
                }

                using (HttpWebResponse response = Response.Http(definition) as HttpWebResponse)
                {
                    this.Visited.Add(uri);
                    XPathNavigator navigator = null;
                    switch (response.StatusCode)
                    {
                        case HttpStatusCode.OK: /// 200 OK
                            if (((Uri)uri).Host.EndsWith(this.Domain, StringComparison.OrdinalIgnoreCase))
                            {
                                Response content = Response.ToTypedContent(response);
                                HtmlResponse html = content as HtmlResponse;
                                if (null != html)
                                {
                                    navigator = html.Html.CreateNavigator();
                                }

                                XmlResponse xml = content as XmlResponse;
                                if (null != xml)
                                {
                                    navigator = xml.Xml.CreateNavigator();
                                }

                                uri = Spider.ResolveBaseUri(uri, navigator);

                                if (response.ContentType.StartsWith("text/html", StringComparison.OrdinalIgnoreCase))
                                {
                                    Spider.CheckHtmlStructure(content, navigator);
                                    Spider.CheckHtmlAccessibility(content, navigator);
                                }
                                else if (response.ContentType.StartsWith("application/xhtml+xml", StringComparison.OrdinalIgnoreCase))
                                {
                                    Spider.CheckXhtmlStructure(content, navigator);
                                    Spider.CheckXhtmlAccessibility(content, navigator);
                                }

                                this.Crawl(uri, navigator);
                            }

                            break;

                        case HttpStatusCode.MultipleChoices: /// 300 Multiple Choices
                            AbsoluteUri location = response.Headers[HttpResponseHeader.Location];
                            if (null != location)
                            {
                                this.Crawl(uri, (string)location);
                            }

                            break;

                        case HttpStatusCode.Moved: /// 301 Moved Permanently
                            this.Crawl(uri, (string)response.Headers[HttpResponseHeader.Location]);
                            break;

                        case HttpStatusCode.Found: /// 302 Found
                            this.Crawl(uri, (string)response.Headers[HttpResponseHeader.Location]);
                            break;

                        case HttpStatusCode.SeeOther: /// 303 See Other
                            this.Crawl(uri, (string)response.Headers[HttpResponseHeader.Location]);
                            break;

                        case HttpStatusCode.Unauthorized: /// 401 Unauthorized
                            break;

                        default:
                            throw new HttpTestException(Response.ToTypedContent(response), "The spider encountered an unexpected response status code.");
                    }
                }
            }

            return true;
        }

        private bool Crawl(AbsoluteUri uri, XPathNavigator navigator)
        {
            if (null != navigator)
            {
                XPathNodeIterator iterator = null;

                /// first, follow text/html links:

                this.Crawl(uri, navigator.Select("//a[@href][not(@rel='nofollow')]/@href"));
                this.Crawl(uri, navigator.Select("//blockquote[@cite]/@cite"));
                this.Crawl(uri, navigator.Select("//del[@cite]/@cite"));
                this.Crawl(uri, navigator.Select("//frame[@longdesc]/@longdesc"));
                this.Crawl(uri, navigator.Select("//frame[@src]/@src"));
                this.Crawl(uri, navigator.Select("//iframe[@longdesc]/@longdesc"));
                this.Crawl(uri, navigator.Select("//iframe[@src]/@src"));
                this.Crawl(uri, navigator.Select("//img[@src]/@src"));
                this.Crawl(uri, navigator.Select("//img[@longdesc]/@longdesc"));
                this.Crawl(uri, navigator.Select("//input[@src]/@src"));
                this.Crawl(uri, navigator.Select("//ins[@cite]/@cite"));
                this.Crawl(uri, navigator.Select("//object[@archive]/@archive"), ' ');
                this.Crawl(uri, navigator.Select("//object[@codebase]/@codebase"));
                this.Crawl(uri, navigator.Select("//object[@data]/@data"));
                this.Crawl(uri, navigator.Select("//object[@usemap]/@usemap"));
                this.Crawl(uri, navigator.Select("//q[@cite]/@cite"));

                this.Crawl(uri, navigator.Select("//link[@href][not(@rel='nofollow')][not(@type)]/@href"));
                iterator = navigator.Select("//link[@href][not(@rel='nofollow')][@type]");
                while (iterator.MoveNext())
                {
                    this.Crawl(
                        uri,
                        (string)iterator.Current.GetAttribute("href", null),
                        new ContentType(iterator.Current.GetAttribute("type", null)));
                }

                /// next, follow xhtml links:

                XmlNamespaceManager namespaces = new XmlNamespaceManager(navigator.NameTable);
                namespaces.AddNamespace("xhtml", xhtml);

                this.Crawl(uri, navigator.Select("//xhtml:a[@href][not(@rel='nofollow')]/@href", namespaces));
                this.Crawl(uri, navigator.Select("//xhtml:blockquote[@cite]/@cite", namespaces));
                this.Crawl(uri, navigator.Select("//xhtml:del[@cite]/@cite", namespaces));
                this.Crawl(uri, navigator.Select("//xhtml:frame[@longdesc]/@longdesc", namespaces));
                this.Crawl(uri, navigator.Select("//xhtml:frame[@src]/@src", namespaces));
                this.Crawl(uri, navigator.Select("//xhtml:iframe[@longdesc]/@longdesc", namespaces));
                this.Crawl(uri, navigator.Select("//xhtml:iframe[@src]/@src", namespaces));
                this.Crawl(uri, navigator.Select("//xhtml:img[@src]/@src", namespaces));
                this.Crawl(uri, navigator.Select("//xhtml:img[@longdesc]/@longdesc", namespaces));
                this.Crawl(uri, navigator.Select("//xhtml:input[@src]/@src", namespaces));
                this.Crawl(uri, navigator.Select("//xhtml:ins[@cite]/@cite", namespaces));
                this.Crawl(uri, navigator.Select("//xhtml:object[@archive]/@archive", namespaces), ' ');
                this.Crawl(uri, navigator.Select("//xhtml:object[@codebase]/@codebase", namespaces));
                this.Crawl(uri, navigator.Select("//xhtml:object[@data]/@data", namespaces));
                this.Crawl(uri, navigator.Select("//xhtml:object[@usemap]/@usemap", namespaces));
                this.Crawl(uri, navigator.Select("//xhtml:q[@cite]/@cite", namespaces));

                this.Crawl(uri, navigator.Select("//xhtml:link[@href][not(@rel='nofollow')][not(@type)]/@href", namespaces));
                iterator = navigator.Select("//xhtml:link[@href][not(@rel='nofollow')][@type]", namespaces);
                while (iterator.MoveNext())
                {
                    this.Crawl(
                        uri,
                        (string)iterator.Current.GetAttribute("href", string.Empty),
                        new ContentType(iterator.Current.GetAttribute("type", string.Empty)));
                }

                /// xlink

                namespaces.AddNamespace("xlink", xlink);

                this.Crawl(uri, navigator.Select("//*[@xlink:type='simple'][@xlink:href]/@xlink:href", namespaces));

                /// sitemaps

                namespaces.AddNamespace("sitemaps", sitemaps);

                this.Crawl(uri, navigator.Select("//sitemaps:loc[text()]/text()", namespaces));

                /// RDF Site Summary (RSS) 1.0

                namespaces.AddNamespace("dc", dc);
                namespaces.AddNamespace("rdf", rdf);
                namespaces.AddNamespace("rss", rss);

                this.Crawl(uri, navigator.Select("//@rdf:about", namespaces));
                this.Crawl(uri, navigator.Select("//rdf:link[text()]/text()", namespaces));

                /// RSS 2.0

                this.Crawl(uri, navigator.Select("/rss[@version='2.0']/channel/link[text()]/text()"));
                this.Crawl(uri, navigator.Select("/rss[@version='2.0']/channel/item/link[text()]/text()"));

                /// Atom

                namespaces.AddNamespace("atom", atom);

                this.Crawl(uri, navigator.Select("//atom:link[@href][not(@type)]/@href", namespaces));
                iterator = navigator.Select("//atom:link[@href][@type]", namespaces);
                while (iterator.MoveNext())
                {
                    this.Crawl(
                        uri,
                        (string)iterator.Current.GetAttribute("href", string.Empty),
                        new ContentType(iterator.Current.GetAttribute("type", string.Empty)));
                }
            }

            return true;
        }

        private bool Crawl(AbsoluteUri uri, XPathNodeIterator iterator)
        {
            if (null != iterator)
            {
                while (iterator.MoveNext())
                {
                    string href = iterator.Current.Value;
                    if (!string.IsNullOrEmpty(href))
                    {
                        this.Crawl(uri, (string)href);
                    }
                }
            }

            return true;
        }

        private bool Crawl(AbsoluteUri uri, XPathNodeIterator iterator, char separator)
        {
            if (null != iterator)
            {
                while (iterator.MoveNext())
                {
                    string hrefs = iterator.Current.Value;
                    if (!string.IsNullOrEmpty(hrefs))
                    {
                        this.Crawl(uri, hrefs.Split(new char[] { separator }, StringSplitOptions.RemoveEmptyEntries));
                    }
                }
            }

            return true;
        }

        private bool Crawl(AbsoluteUri uri, IEnumerable<string> hrefs)
        {
            foreach (string href in hrefs)
            {
                this.Crawl(uri, (string)href);
            }

            return true;
        }

        private bool Crawl(AbsoluteUri uri, string href)
        {
            return this.Crawl(uri, (string)href, null as ContentType);
        }

        private bool Crawl(AbsoluteUri uri, string href, ContentType type)
        {
            if (null == href)
            {
                throw new ArgumentNullException("href");
            }
            else if (0 == href.Length)
            {
                throw new ArgumentOutOfRangeException("href");
            }

            href = href.Replace("&amp;", "&");

            if (href.StartsWith("#", StringComparison.OrdinalIgnoreCase))
            {
                /// don't follow fragments
            }
            else if (href.Contains(":"))
            {
                if (href.StartsWith("http://", StringComparison.OrdinalIgnoreCase)
                    || href.StartsWith("https://", StringComparison.OrdinalIgnoreCase))
                {
                    this.Crawl((AbsoluteUri)href, type);
                }
                else if (href.StartsWith("mailto:", StringComparison.OrdinalIgnoreCase))
                {
                    if (null == new MailAddress(href.Substring("mailto:".Length)))
                    {
                        /// quick validation check
                    }
                }
                else if (href.StartsWith("feed:", StringComparison.OrdinalIgnoreCase))
                {
                    /// TODO : http://en.wikipedia.org/wiki/URI_scheme
                }
                else if (href.StartsWith("webcal:", StringComparison.OrdinalIgnoreCase))
                {
                    /// TODO : http://en.wikipedia.org/wiki/URI_scheme
                }
            }
            else
            {
                this.Crawl(uri, (RelativeUri)href, type);
            }

            return true;
        }

        private bool Crawl(AbsoluteUri uri, RelativeUri relative, ContentType type)
        {
            return this.Crawl(new Uri(uri, (Uri)relative), type);
        }
    }
}